#!/bin/groovy
/*
 * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The OpenAirInterface Software Alliance licenses this file to You under
 * the OAI Public License, Version 1.1  (the "License"); you may not use this file
 * except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.openairinterface.org/?page_id=698
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *-------------------------------------------------------------------------------
 * For more information about the OpenAirInterface (OAI) Software Alliance:
 *      contact@openairinterface.org
 */

//-------------------------------------------------------------------------------
// Abstraction function to send social media messages:
// like on Slack or Mattermost
def sendSocialMediaMessage(pipeChannel, pipeColor, pipeMessage) {
  if (params.pipelineUsesSlack != null) {
    if (params.pipelineUsesSlack) {
      slackSend channel: pipeChannel, color: pipeColor, message: pipeMessage
    }
  }
}

// Location of the CN executor node
def cn_ci_host = params.Host_CN_CI_Server

// for lock
def cn_ci_resource = params.DockerContainers

// Location of the Remote Ubuntu18 CN executor
def rem_u18_host_flag = false
def rem_u18_host = ""
def rem_u18_host_user = ""

// Location of the Remote RHEL CN executor
def rem_rhel_host_flag = false
def rem_rhel_host = ""
def rem_rhel_host_user = ""
def rem_rhel8_resource = params.PodmanContainers

// Variables to pass to the FED Test job
def spgwu_tag = 'develop'
def spgwu_branch = 'develop'

//-------------------------------------------------------------------------------
// Pipeline start
pipeline {
  agent {
    label cn_ci_host
  }
  options {
    disableConcurrentBuilds()
    timestamps()
    ansiColor('xterm')
    lock(cn_ci_resource)
  }

  stages {
    stage ('Verify Parameters') {
      steps {
        script {
          echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'

          JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"'
          JOB_TIMESTAMP = JOB_TIMESTAMP.trim()

          if (params.Host_CN_CI_2nd_Server_Flag != null) {
            rem_u18_host_flag = params.Host_CN_CI_2nd_Server_Flag
            if (rem_u18_host_flag) {
              rem_u18_host = params.Host_CN_CI_2nd_Server
              rem_u18_host_user = params.Host_CN_CI_2nd_Server_Login
              echo "1st Node   is ${NODE_NAME}"
              echo "2nd Node   is ${rem_u18_host}"
            } else {
              echo "U18 Node   is ${NODE_NAME}"
            }
          } else {
            echo "Node       is ${NODE_NAME}"
          }
          if (params.Remote_RHEL_Server_Flag != null) {
            rem_rhel_host_flag = params.Remote_RHEL_Server_Flag
            if (rem_rhel_host_flag) {
              rem_rhel_host = params.Remote_RHEL_Server_Name
              rem_rhel_host_user = params.Remote_RHEL_Server_Login
              echo "RHEL Node  is ${rem_rhel_host}"
            }
          }
          echo "Git URL    is ${GIT_URL}"
          if (params.DockerHubCredentials == null) {
            echo '\u26D4 \u001B[31mNo Credentials to push to DockerHub!\u001B[0m'
            error "Stopping pipeline!"
          }

          removeGitHubInfo()
          if (env.ghprbPullId != null) {
            echo "======= THIS IS A PULL REQUEST ======"
            echo "PR ID      is ${env.ghprbPullId}"
            echo "PR LINK    is ${env.ghprbPullLink}"
            echo "PR TITLE   is ${env.ghprbPullTitle}"
            sh "git fetch --prune --unshallow"
            shortenShaOne = sh returnStdout: true, script: 'git log -1 --pretty=format:"%h" ' + env.ghprbActualCommit
            shortenShaOne = shortenShaOne.trim()
            sh "./ci-scripts/doGitHubPullRequestTempMerge.sh --src-branch ${env.ghprbSourceBranch} --src-commit ${env.ghprbActualCommit} --target-branch ${env.ghprbTargetBranch} --target-commit ${GIT_COMMIT}"
            sh "sleep 10"
            spgwu_tag = 'ci-temp'
            rhel_spgwu_tag = 'ci-temp-pr-' + env.ghprbPullId + '-' + shortenShaOne
            spgwu_branch = env.ghprbSourceBranch
            echo "PR TAG     is ${rhel_spgwu_tag}"
          } else {
            echo "======= THIS IS A PUSH EVENT ======"
            echo "Git Branch is ${GIT_BRANCH}"
            echo "Git Commit is ${GIT_COMMIT}"
            // Making a hard copy of the SHA-ONE
            getCommitSha()
          }
        }
      }
    }
    stage ('Prepare Source Code') {
      steps {
        script {
          sh "git clean -x -d -f > /dev/null 2>&1"
          sh "tar -cjhf /tmp/openair-spgwu.tar.bz2 ."
          sh "mv /tmp/openair-spgwu.tar.bz2 ."
          copyTo2ndServer('openair-spgwu.tar.bz2', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          copyTo2ndServer('openair-spgwu.tar.bz2', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
          sh "mkdir -p archives/oai-spgwu-cfg"
        }
      }
    }
    stage('Build Core Network Function') {
      parallel {
        stage ('Build U18 SPGW-U Image') {
          steps {
            script {
              if (env.ghprbPullId == null) {
                // Currently this pipeline only runs for pushes to `develop` branch
                // First clean image registry
                myShCmd('docker image rm oai-spgwu-tiny:develop || true', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              } else {
                // In case we forgot during a previous run
                myShCmd('docker image rm oai-spgwu-tiny:' + spgwu_tag + ' || true', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              }
              myShCmd('docker image prune --force', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd('docker build --no-cache --target oai-spgwu-tiny --tag oai-spgwu-tiny:' + spgwu_tag + ' --file docker/Dockerfile.ubuntu18.04 --build-arg EURECOM_PROXY="http://proxy.eurecom.fr:8080" . > archives/spgwu_docker_image_build.log 2>&1', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd('docker image ls >> archives/spgwu_docker_image_build.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              // We will keep also the TMP image in U18 case.
              if (env.ghprbPullId != null) {
                myShCmd('docker image tag oai-spgwu-tiny:' + spgwu_tag + ' oai-spgwu-tiny:' + rhel_spgwu_tag, rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              }
            }
          }
          post {
            always {
              script {
                copyFrom2ndServer('archives/spgwu_docker_image_build.log', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              }
            }
            success {
              sh "echo 'OAI-SPGW-U DOCKER IMAGE BUILD: OK' >> archives/spgwu_docker_image_build.log"
            }
            unsuccessful {
              sh "echo 'OAI-SPGW-U DOCKER IMAGE BUILD: KO' >> archives/spgwu_docker_image_build.log"
            }
          }
        }
        stage ('Build RHEL8 SPGW-U Image') {
          when { expression {rem_rhel_host_flag} }
          steps {
            lock (rem_rhel8_resource) {
              script {
                if (env.ghprbPullId == null) {
                  // Currently this pipeline only runs for pushes to `develop` branch
                  // First clean image registry
                  myShCmd('sudo podman image rm oai-spgwu-tiny:develop || true', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                } else {
                  // In case we forgot during a previous run
                  myShCmd('sudo podman image rm oai-spgwu-tiny:' + spgwu_tag + ' || true', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                }
                myShCmd('sudo podman image prune --force', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                // Copy the RHEL Host certificates for building
                myShCmd('mkdir -p tmp/ca tmp/entitlement', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                myShCmd('cp /etc/pki/entitlement/*pem tmp/entitlement', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                myShCmd('sudo cp /etc/rhsm/ca/redhat-uep.pem tmp/ca', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                myShCmd('sudo podman build --no-cache --target oai-spgwu-tiny --tag oai-spgwu-tiny:' + spgwu_tag + ' --file docker/Dockerfile.rhel8  --build-arg EURECOM_PROXY="http://proxy.eurecom.fr:8080" . > archives/spgwu_podman_image_build.log 2>&1', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                myShCmd('sudo podman image ls >> archives/spgwu_podman_image_build.log', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                if (env.ghprbPullId != null) {
                  myShCmd('sudo podman image tag oai-spgwu-tiny:' + spgwu_tag + ' oai-spgwu-tiny:' + rhel_spgwu_tag, rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
                }
              }
            }
          }
          post {
            always {
              script {
                copyFrom2ndServer('archives/spgwu_podman_image_build.log', 'archives', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
              }
            }
            success {
              sh "echo 'OAI-SPGW-U PODMAN RHEL8 IMAGE BUILD: OK' >> archives/spgwu_podman_image_build.log"
            }
            unsuccessful {
              sh "echo 'OAI-SPGW-U PODMAN RHEL8 IMAGE BUILD: KO' >> archives/spgwu_podman_image_build.log"
            }
          }
        }
        // Running CPPCHECK in parallel to gain time
        stage ('Static Code Analysis') {
          steps {
            script {
              // Running on xenial to have 1.72 version of cppcheck
              myShCmd('docker run --name ci-cn-cppcheck -d ubuntu:xenial /bin/bash -c "sleep infinity"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "apt-get update && apt-get upgrade --yes" > archives/cppcheck_install.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "apt-get install --yes cppcheck bzip2" >> archives/cppcheck_install.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)

              myShCmd('docker cp ./openair-spgwu.tar.bz2 ci-cn-cppcheck:/home', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "cd /home && tar -xjf openair-spgwu.tar.bz2"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "rm -f /home/openair-spgwu.tar.bz2"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)

              myShCmd('docker exec ci-cn-cppcheck /bin/bash -c "cd /home && cppcheck --enable=warning --force --xml --xml-version=2 src 2> cppcheck.xml 1> cppcheck_build.log"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
            }
          }
          post {
            always {
              script {
                myShCmd('docker cp ci-cn-cppcheck:/home/cppcheck.xml archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                myShCmd('docker cp ci-cn-cppcheck:/home/cppcheck_build.log archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                copyFrom2ndServer('archives/cppcheck*.*', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                // no need to keep the cppcheck container
                myShCmd('docker rm -f ci-cn-cppcheck', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              }
            }
            success {
              sh "echo 'CPPCHECK: OK' >> archives/cppcheck_install.log"
            }
            unsuccessful {
              sh "echo 'CPPCHECK: KO' >> archives/cppcheck_install.log"
            }
          }
        }
        // Running CLANG-FORMATTING check in parallel to gain time
        stage ('Code Formatting Checker') {
          steps {
            script {
              myShCmd('docker run --name ci-cn-clang-formatter -d ubuntu:bionic /bin/bash -c "sleep infinity"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "apt-get update && apt-get upgrade --yes" > archives/clang_format_install.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "apt-get install --yes git tree bzip2" >> archives/clang_format_install.log', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)

              myShCmd('docker cp ./openair-spgwu.tar.bz2 ci-cn-clang-formatter:/home', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "cd /home && tar -xjf openair-spgwu.tar.bz2"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "rm -f /home/openair-spgwu.tar.bz2"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)

              // We install a dedicated version (installed on our CI server).
              myShCmd('docker cp /opt/clang-format/9.0.0/bin/clang-format ci-cn-clang-formatter:/usr/local/bin', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              if (env.ghprbPullId != null) {
                myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "cd /home && ./ci-scripts/checkCodingFormattingRules.sh --src-branch ' + env.ghprbSourceBranch +' --target-branch ' + env.ghprbTargetBranch + '"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              } else {
                myShCmd('docker exec ci-cn-clang-formatter /bin/bash -c "cd /home && ./ci-scripts/checkCodingFormattingRules.sh"', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              }
            }
          }
          post {
            always {
              script {
                myShCmd('docker cp ci-cn-clang-formatter:/home/src/oai_rules_result.txt src', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                // May not have been generated
                try {
                  myShCmd('docker cp ci-cn-clang-formatter:/home/src/oai_rules_result_list.txt src', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                } catch (Exception e) {
                  echo "Failed to copy src/oai_rules_result_list.txt! It may not have been generated. That's OK!"
                }
                copyFrom2ndServer('archives/clang_format*.*', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                copyFrom2ndServer('src/oai_rules*.*', 'src', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                // no need to keep the clang-formatter container
                myShCmd('docker rm -f ci-cn-clang-formatter', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              }
            }
          }
        }
      }
    }
    stage('Deploy Sanity Check Test') {
      stages {
        stage('Create Docker Networks') {
          steps {
            script {
              myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=CreateNetworks', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
            }
          }
        }
        stage('Deploy OAI-SPGW-C') {
          steps {
            script {
              myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=DeploySPGWC --tag=develop', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
            }
          }
          post {
            always {
              script {
                copyFrom2ndServer('archives/spgwc_config.log', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              }
            }
            success {
              sh "echo 'OAI-SPGW-C CONFIG: OK' >> archives/spgwc_config.log"
            }
            unsuccessful {
              sh "echo 'OAI-SPGW-C CONFIG: KO' >> archives/spgwc_config.log"
            }
          }
        }
        stage('Deploy OAI-SPGWU-TINY') {
          steps {
            script {
              myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=DeploySPGWU --tag=' + spgwu_tag, rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
            }
          }
          post {
            always {
              script {
                myShCmd('docker cp ci-oai-spgwu:/openair-spgwu-tiny/etc/. archives/oai-spgwu-cfg', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                copyFrom2ndServer('archives/spgwu_config.log', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
                copyFrom2ndServer('archives/oai-spgwu-cfg/*.*', 'archives/oai-spgwu-cfg', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              }
            }
            success {
              sh "echo 'OAI-SPGW-U CONFIG: OK' >> archives/spgwu_config.log"
            }
            unsuccessful {
              sh "echo 'OAI-SPGW-U CONFIG: KO' >> archives/spgwu_config.log"
            }
          }
        }
      }
    }
    stage ('Start-Check-Stop OAI cNFs') {
      steps {
        script {
          myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=StartSPGWC --tag=develop', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          sh "sleep 2"
          myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=StartSPGWU --tag=' + spgwu_tag, rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          sh "sleep 40"
          myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=StopSPGWC --tag=develop', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=StopSPGWU --tag=' + spgwu_tag, rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          sh "sleep 2"
          myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=RetrieveLogsSPGWC --tag=develop', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=RetrieveLogsSPGWU --tag=' + spgwu_tag, rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          copyFrom2ndServer('archives/*_check_run.log', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          sh "python3 ./ci-scripts/verifySanityCheckDeployment.py --job_name=${JOB_NAME} --job_id=${BUILD_ID}"
        }
      }
      post {
        success {
          sh "echo 'OAI-SPGW-C CHECK START/STOP: OK' > archives/spgwc_cnf_check_start.log"
          sh "echo 'OAI-SPGW-U CHECK START/STOP: OK' > archives/spgwu_cnf_check_start.log"
        }
        unsuccessful {
          // If anything wrong occurs, we still try to copy
          myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=RetrieveLogsSPGWC --tag=develop', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=RetrieveLogsSPGWU --tag=' + spgwu_tag, rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          copyFrom2ndServer('archives/*_check_run.log', 'archives', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          sh "echo 'OAI-SPGW-C CHECK START/STOP: KO' > archives/spgwc_cnf_check_start.log"
          sh "echo 'OAI-SPGW-U CHECK START/STOP: KO' > archives/spgwu_cnf_check_start.log"
        }
      }
    }
    stage ('Undeploy Sanity Check Test') {
      steps {
        script {
          // Killing all containers
          myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=RemoveAllContainers', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          // Removing all intermediate networks
          myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=RemoveNetworks', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
        }
      }
    }
    stage ('Testing Images') {
      parallel {
        stage ('Testing in FED environment') {
          steps {
            script {
              localStatus = build job: params.CN_EPC_FED_PipelineName,
                parameters: [
                  string(name: 'SPGWU_TAG', value: String.valueOf(spgwu_tag)),
                  string(name: 'SPGWU_BRANCH', value: String.valueOf(spgwu_branch))
                ], propagate: false
              localResult = localStatus.getResult()

              if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) {
                echo "Federation Test Job is OK"
              } else {
                echo "Federation Test Job is KO"
                sh "ci-scripts/fail.sh"
              }
            }
          }
          post {
            always {
              script {
                copyArtifacts(projectName: params.CN_EPC_FED_PipelineName,
                              filter: 'test_results_oai_epc.html',
                              selector: lastCompleted())
              }
            }
          }
        }
        stage ('RHEL8 Sanity Check Deployment') {
          when { expression {rem_rhel_host_flag} }
          steps {
            script {
              localStatus = build job: params.RHEL8_SanityCheck_PipelineName,
                parameters: [
                  string(name: 'SPGWU_TAG', value: String.valueOf(spgwu_tag)),
                  string(name: 'SPGWU_BRANCH', value: String.valueOf(spgwu_branch))
                ], propagate: false
              localResult = localStatus.getResult()

              if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) {
                echo "RHEL8 Sanity Check Deployment Test Job is OK"
              } else {
                echo "RHEL8 Sanity Check Deployment Test Job is KO"
                sh "ci-scripts/fail.sh"
              }
            }
          }
          post {
            always {
              script {
                copyArtifacts(projectName: params.RHEL8_SanityCheck_PipelineName,
                              filter: 'test_results_magma_epc_rhel8.html',
                              selector: lastCompleted())
              }
            }
          }
        }
        stage ('Testing in 5G FED Docker environment') {
          steps {
            script {
              localStatus = build job: params.CN5G_FED_PipelineName,
                parameters: [
                  string(name: 'SPGWU_TAG', value: String.valueOf(spgwu_tag)),
                  string(name: 'SPGWU_BRANCH', value: String.valueOf(spgwu_branch))
                ], propagate: false
              localResult = localStatus.getResult()

              if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) {
                echo "Docker Federation Test Job is OK"
              } else {
                echo "Docker Federation Test Job is KO"
                sh "ci-scripts/fail.sh"
              }
            }
          }
          post {
            always {
              script {
                copyArtifacts(projectName: params.CN5G_FED_PipelineName,
                              filter: 'test_results_oai_cn5g.html',
                              selector: lastCompleted())
              }
            }
          }
        }
        stage ('Testing in 5G FED OC environment') {
          steps {
            script {
              localStatus = build job: params.CN5G_FED_OC_PipelineName,
                parameters: [
                  string(name: 'SPGWU_TAG', value: String.valueOf(spgwu_tag)),
                  string(name: 'SPGWU_BRANCH', value: String.valueOf(spgwu_branch))
                ], propagate: false
              localResult = localStatus.getResult()

              if (localStatus.resultIsBetterOrEqualTo('SUCCESS')) {
                echo "OC Federation Test Job is OK"
              } else {
                echo "OC Federation Test Job is KO"
                sh "ci-scripts/fail.sh"
              }
            }
          }
          post {
            always {
              script {
                copyArtifacts(projectName: params.CN5G_FED_OC_PipelineName,
                              filter: 'test_results_oai_cn5g_oc.html',
                              selector: lastCompleted())
              }
            }
          }
        }
      }
    }
    // For the moment it is Docker-Hub, but we might have a new one internally.
    stage ('Pushing Image to Official Registry') {
      steps {
        script {
          // Only in case of push to target branch!
          if (env.ghprbPullId == null) {
            withCredentials([
              [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.DockerHubCredentials}", usernameVariable: 'DH_Username', passwordVariable: 'DH_Password']
            ]) {
              myShCmd("echo ${DH_Password} | docker login --username ${DH_Username} --password-stdin", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd("docker image tag oai-spgwu-tiny:develop ${DH_Username}/oai-spgwu-tiny:develop", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd("docker push ${DH_Username}/oai-spgwu-tiny:develop", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd("docker rmi ${DH_Username}/oai-spgwu-tiny:develop", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
              myShCmd("docker logout", rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
            }
          }
        }
      }
    }
  }
  post {
    success {
      script {
        def message = "Build job #" + BUILD_ID + " completed"
        updateGithubCommitStatus(currentBuild, message)
        def socialMessage = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): passed (" + BUILD_URL + ")"
        sendSocialMediaMessage('ci-epc', 'good', socialMessage)
      }
    }
    failure {
      script {
        def message = "Build job #" + BUILD_ID + " : One step of job failed"
        updateGithubCommitStatus(currentBuild, message)
        def socialMessage = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): failed (" + BUILD_URL + ")"
        sendSocialMediaMessage('ci-epc', 'danger', socialMessage)
      }
    }
    unstable {
      script {
        def message = "Build job #" + BUILD_ID + " : One step of job is unstable"
        updateGithubCommitStatus(currentBuild, message)
        def socialMessage = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): unstable (" + BUILD_URL + ")"
        sendSocialMediaMessage('ci-epc', 'danger', socialMessage)
      }
    }
    aborted {
      script {
        def message = "Build job #" + BUILD_ID + " aborted: pipeline script issue?"
        updateGithubCommitStatus(currentBuild, message)
        def socialMessage = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): aborted (" + BUILD_URL + ")"
        sendSocialMediaMessage('ci-epc', 'danger', socialMessage)
      }
    }
    cleanup {
      script {
        // Killing all containers
        myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=RemoveAllContainers', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
        // Removing the networks
        myShCmd('python3 ./ci-scripts/sanityCheckDeploy.py --action=RemoveNetworks', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
        // In case of build failure, we might have temporary containers still running. TBD!

        // Removing temporary / intermediate images
        try {
          if (env.ghprbPullId != null) {
            myShCmd('docker image rm --force oai-spgwu-tiny:ci-temp', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
          }
        } catch (Exception e) {
          echo "We failed to delete the OAI-SPGWU temp image"
        }
        try {
          if (env.ghprbPullId != null) {
            myShCmd('sudo podman image rm oai-spgwu-tiny:ci-temp', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
          }
        } catch (Exception e) {
          echo "We failed to delete the OAI-SPGWU temp image"
        }
        try {
          myShCmd('docker image prune --force', rem_u18_host_flag, rem_u18_host_user, rem_u18_host)
        } catch (Exception e) {
          echo "We failed to prune all unneeded intermediate images"
        }
        if (rem_rhel_host_flag) {
          try {
            myShCmd('sudo podman image prune --force', rem_rhel_host_flag, rem_rhel_host_user, rem_rhel_host)
          } catch (Exception e) {
            echo "We failed to prune all unneeded intermediate images"
          }
        }

        // Zipping all archived log files
        sh "zip -r -qq docker_logs.zip archives"
        if (fileExists('docker_logs.zip')) {
          archiveArtifacts artifacts: 'docker_logs.zip'
        }
        // Generating the HTML report
        if (env.ghprbPullId != null) {
          sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${env.ghprbSourceBranch} --git_src_commit=${env.ghprbActualCommit} --git_pull_request=True --git_target_branch=${env.ghprbTargetBranch} --git_target_commit=${GIT_COMMIT}"
          if (fileExists('test_results_oai_spgwu.html')) {
            sh "sed -i -e 's#TEMPLATE_PULL_REQUEST_LINK#${env.ghprbPullLink}#g' test_results_oai_spgwu.html"
            sh "sed -i -e 's#TEMPLATE_PULL_REQUEST_TEMPLATE#${env.ghprbPullTitle}#' test_results_oai_spgwu.html"
          }
          if (fileExists('test_results_oai_epc.html')) {
            sh "sed -i -e 's#TEMPLATE_PULL_REQUEST_LINK#${env.ghprbPullLink}#g' test_results_oai_epc.html"
            sh "sed -i -e 's#TEMPLATE_PULL_REQUEST_TEMPLATE#${env.ghprbPullTitle}#' test_results_oai_epc.html"
          }
          if (fileExists('test_results_magma_epc_rhel8.html')) {
            sh "sed -i -e 's#TEMPLATE_PULL_REQUEST_LINK#${env.ghprbPullLink}#g' test_results_magma_epc_rhel8.html"
            sh "sed -i -e 's#TEMPLATE_PULL_REQUEST_TEMPLATE#${env.ghprbPullTitle}#' test_results_magma_epc_rhel8.html"
          }
          if (fileExists('test_results_oai_cn5g.html')) {
            sh "sed -i -e 's#TEMPLATE_PULL_REQUEST_LINK#${env.ghprbPullLink}#g' test_results_oai_cn5g.html"
            sh "sed -i -e 's#TEMPLATE_PULL_REQUEST_TEMPLATE#${env.ghprbPullTitle}#' test_results_oai_cn5g.html"
          }
          if (fileExists('test_results_oai_cn5g_oc.html')) {
            sh "sed -i -e 's#TEMPLATE_PULL_REQUEST_LINK#${env.ghprbPullLink}#g' test_results_oai_cn5g_oc.html"
            sh "sed -i -e 's#TEMPLATE_PULL_REQUEST_TEMPLATE#${env.ghprbPullTitle}#' test_results_oai_cn5g_oc.html"
          }
        } else {
          sh "python3 ci-scripts/generateHtmlReport.py --job_name=${JOB_NAME} --job_id=${BUILD_ID} --job_url=${BUILD_URL} --git_url=${GIT_URL} --git_src_branch=${GIT_BRANCH} --git_src_commit=${GIT_COMMIT}"
        }
        if (fileExists('test_results_oai_spgwu.html')) {
          sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' test_results_oai_spgwu.html"
          archiveArtifacts artifacts: 'test_results_oai_spgwu.html'
        }
        if (fileExists('test_results_oai_epc.html')) {
          sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' test_results_oai_epc.html"
          archiveArtifacts artifacts: 'test_results_oai_epc.html'
        }
        if (fileExists('test_results_magma_epc_rhel8.html')) {
          sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' test_results_magma_epc_rhel8.html"
          archiveArtifacts artifacts: 'test_results_magma_epc_rhel8.html'
        }
        if (fileExists('test_results_oai_cn5g.html')) {
          sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' test_results_oai_cn5g.html"
          archiveArtifacts artifacts: 'test_results_oai_cn5g.html'
        }
        if (fileExists('test_results_oai_cn5g_oc.html')) {
          sh "sed -i -e 's#TEMPLATE_TIME#${JOB_TIMESTAMP}#' test_results_oai_cn5g_oc.html"
          archiveArtifacts artifacts: 'test_results_oai_cn5g_oc.html'
        }
        // Sending an email to the last committer. Maybe not the Pull Request's author.
        if (params.sendToCommitterEmail != null) {
          if (params.sendToCommitterEmail) {
            def emailTo = getCommitterEmail()
            emailext attachmentsPattern: '*results*.html',
              body: '''Hi,
Here are attached HTML report files for $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!

Regards,
OAI CI Team''',
              replyTo: 'no-reply@openairinterface.org',
              subject: '$PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!',
              to: emailTo
          }
        }
      }
    }
  }
}

// Removing all info -- do not know if git clean command removes them
def removeGitHubInfo() {
  if (fileExists('.git/remote-url')) {
    sh "rm -f .git/remote-url"
  }
  if (fileExists('.git/current-commit')) {
    sh "rm -f .git/current-commit"
  }
  if (fileExists('.git/commit-email')) {
    sh "rm -f .git/commit-email"
  }
}

// GitHub retrieve email of last committer
def getCommitterEmail() {
  if (!fileExists('.git/commit-email')) {
    sh "git log -1 --pretty=format:\"%ce\" > .git/commit-email"
    // Workaround for issue when merge to develop from the Pull Request GitHub webpage
    // Click on merge button creates a commit with an email address in noreply@github.com
    sh "if [ `egrep -c \"noreply@github.com\" .git/commit-email` -eq 1 ]; then echo \"raphael.defosseux@openairinterface.org\" > .git/commit-email; fi"
  }
  return readFile(".git/commit-email").trim()
}

// Github commit update functions
def getRepoURL() {
  if (!fileExists('.git/remote-url')) {
    sh "git config --get remote.origin.url > .git/remote-url"
  }
  return readFile(".git/remote-url").trim()
}

def getCommitSha() {
  if (!fileExists('.git/current-commit')) {
    sh "git rev-parse HEAD > .git/current-commit"
  }
  return readFile(".git/current-commit").trim()
}

def updateGithubCommitStatus(build, jobMessage) {
  // workaround https://issues.jenkins-ci.org/browse/JENKINS-38674
  repoUrl = getRepoURL()
  commitSha = getCommitSha()

  step([
    $class: 'GitHubCommitStatusSetter',
    reposSource: [$class: "ManuallyEnteredRepositorySource", url: repoUrl],
    commitShaSource: [$class: "ManuallyEnteredShaSource", sha: commitSha],
    errorHandlers: [[$class: 'ShallowAnyErrorHandler']],
    statusResultSource: [
      $class: 'ConditionalStatusResultSource',
      results: [
        [$class: 'AnyBuildResult', state: build.currentResult, message: jobMessage]
      ]
    ]
  ])
}

def copyTo2ndServer(filename, flag, user, host) {
  if (flag) {
    if ("openair-spgwu.tar.bz2".equals(filename)) {
      sh "ssh ${user}@${host} 'rm -rf /tmp/CI-CN-SPGWU'"
      sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN-SPGWU'"
    }
    sh "scp ${filename} ${user}@${host}:/tmp/CI-CN-SPGWU"
    if ("openair-spgwu.tar.bz2".equals(filename)) {
      sh "ssh ${user}@${host} 'cd /tmp/CI-CN-SPGWU && tar -xjf ${filename}'"
      sh "ssh ${user}@${host} 'mkdir -p /tmp/CI-CN-SPGWU/archives/oai-spgwu-cfg'"
      sh "ssh ${user}@${host} 'rm -Rf /tmp/CI-CN-SPGWU/${filename}'"
    }
  }
}

def copyFrom2ndServer(filename, target, flag, user, host) {
  if (flag) {
    sh "scp ${user}@${host}:/tmp/CI-CN-SPGWU/${filename} ${target}"
  }
}

def myShCmd(cmd, flag, user, host) {
  if (flag) {
    sh "ssh -t -t ${user}@${host} 'cd /tmp/CI-CN-SPGWU && ${cmd}'"
  } else {
    sh "${cmd}"
  }
}

def myShRetCmd(cmd, flag, user, host) {
  if (flag) {
    ret = sh returnStdout: true, script: "ssh -t -t ${user}@${host} 'cd /tmp/CI-CN-SPGWU && ${cmd}'"
  } else {
    ret = sh returnStdout: true, script: "${cmd}"
  }
  ret = ret.trim()
  return ret
}
